# -*- coding: utf-8 -*-
"""
爬取策略：
1.部分页面需要登录才能访问
2.post方式提交需要找对接口，不然访问不到
3.json数据post时，要注意转化
4.页面中文乱码需要注意页面的编码格式
5.中文可以通过第三方的包转化成拼音
"""

import json
import datetime
import time
import scrapy
from xpinyin import Pinyin
from xiecheng.items import XiechengItem


class XiechengSpider(scrapy.Spider):
    name = 'xiecheng'
    allowed_domains = ['ctrip.com']

    # start_urls = ['http://trains.ctrip.com/']

    def __init__(self):
        super(XiechengSpider, self).__init__()
        self.post_url = 'http://trains.ctrip.com/TrainBooking/Ajax/SearchListHandler.ashx?Action=getSearchList'
        self.post_data = {}

    def bind_data(self, t=('shanghai', 'nanjing', '上海', '南京', '2017-11-25', '2017-11-25')):
        _con = '{{"IsBus":false,"Filter":"0","Catalog":"","IsGaoTie":false,"IsDongChe":false,"CatalogName":"","DepartureCity":"{0}","ArrivalCity":"{1}","HubCity":"","DepartureCityName":"{2}","ArrivalCityName":"{3}","DepartureDate":"{4}","DepartureDateReturn":"{5}","ArrivalDate":"","TrainNumber":""}}'.format(*t)
        self.post_data['value'] = _con

    def start_requests(self):
        self.bind_data()

        yield scrapy.FormRequest(url=self.post_url, formdata=self.post_data, callback=self.parse)

    def parse(self, response):
        for _data in self.generate_post_data():
            self.bind_data(_data)

            yield scrapy.FormRequest(url=self.post_url, formdata=self.post_data, callback=self.parse_item,
                                     meta={"date": _data[-1]})

    def generate_date(self):
        _dates = [(datetime.datetime.now() + datetime.timedelta(days=i)).strftime("%Y-%m-%d") for i in range(1, 2)]

        return _dates

    def generate_city(self):
        _p = Pinyin()
        _lists = ('上海', '南京', '北京')
        _cities = [
            (_p.get_pinyin(_l1, ''), _p.get_pinyin(_l2, ''), _l1, _l2) for _l1 in _lists for _l2 in _lists if _l1 != _l2
        ]

        return _cities

    def generate_post_data(self):
        _post_data = [(*_city, _dates, _dates) for _city in self.generate_city() for _dates in self.generate_date()]

        return _post_data

    def parse_item(self, response):
        _train_date = response.meta['date']
        _dict_data = json.loads(response.body.decode(encoding='gb2312'))
        # if _dict_data['IsShowSnatch'] == True:
        for _items in _dict_data['TrainItemsList']:
            _item = XiechengItem()
            _item['train_name'] = _items['TrainName']
            _item['train_Type'] = _items['TrainType']
            _item['start_station_name'] = _items['StartStationName']
            _item['end_station_name'] = _items['EndStationName']
            _item['strat_time'] = _items['StratTime']
            _item['end_time'] = _items['EndTime']
            _item['use_time'] = _items['UseTime']
            _item['pre_sale_time'] = _items['PreSaleDay']
            _item['train_min_price'] = _items['TrainMinPrice']
            _item['train_date'] = _train_date
            _item['collect_time'] = time.ctime()

            yield _item
